import requests
from lxml import etree
import csv


def spider_information(region: str, career: str, filepath: str, page=10):
    career_names = []  # 岗位名称
    career_salaries = []  # 岗位薪资
    career_educations = []  # 岗位要求学历
    career_exps = []  # 岗位要求经验
    company_names = []  # 公司名称
    company_sizes = []  # 公司规模
    company_natures = []  # 公司性质
    company_types = []  # 岗位类型
    career_mean_salaries = []  # 岗位平均薪资
    for i in range(1, page+1):
        url = f"https://sou.zhaopin.com/?jl={region}&kw={career}&p={i}"
        header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 SLBrowser/9.0.3.1311 SLBChan/105"}
        response = requests.get(url, headers=header)
        e = etree.HTML(response.text)

        all_job = e.xpath('//*[@id="positionList-hook"]/div')

        for job in all_job:
            # 岗位名称
            career_name = job.xpath("./div[1]/div/div/div[1]/div/a/text()")
            # print(career_name)
            career_names.extend(career_name)
            # print(career_names)

            # 岗位薪资
            career_salary = job.xpath("./div[1]/div/div/div[1]/div/p/text()")
            # print(career_salary)
            for j in range(len(career_salary)):
                career_salary[j] = career_salary[j].split()[0]
            # print(career_salary)
            career_salaries.extend(career_salary)
            # print(career_salaries)

            # 岗位要求学历
            career_edu = []
            for j in range(1, len(career_name)+1):
                try:
                    edu = job.xpath(f"./div[{j}]/div[1]/div[1]/div[4]/div[3]/text()")[0]
                    # print(edu)
                    career_edu.append(edu)
                except IndexError:
                    try:
                        edu = job.xpath(f"./div[{j}]/div[1]/div[1]/div[3]/div[3]/text()")[0]
                        # print(edu)
                        career_edu.append(edu)
                    except IndexError:
                        edu = job.xpath(f"./div[{j}]/div[1]/div[1]/div[2]/div[3]/text()")[0]
                        # print(edu)
                        career_edu.append(edu)

            # print(career_edu)
            for j in range(len(career_edu)):
                career_edu[j] = career_edu[j].split()[0]
            career_educations.extend(career_edu)
            # print(career_educations)

            # 岗位要求经验
            career_exp = []
            for j in range(1, len(career_name)+1):
                try:
                    exp = job.xpath(f"./div[{j}]/div[1]/div[1]/div[4]/div[2]/text()")[0]
                    career_exp.append(exp)
                except IndexError:
                    try:
                        exp = job.xpath(f"./div[{j}]/div[1]/div[1]/div[3]/div[2]/text()")[0]
                        career_exp.append(exp)
                    except IndexError:
                        exp = job.xpath(f"./div[{j}]/div[1]/div[1]/div[2]/div[2]/text()")[0]
                        # print(edu)
                        career_exp.append(exp)

            # print(career_exp)
            for j in range(len(career_exp)):
                career_exp[j] = career_exp[j].split()[0]
            # print(career_exp)
            career_exps.extend(career_exp)
            # print(career_exps)

            # 公司名称
            company_name = []
            for j in range(1, len(career_name) + 1):
                try:
                    name = job.xpath(f"./div[{j}]/div[1]/div[2]/div[1]/a/text()")[0]
                    company_name.append(name)
                except IndexError:
                    name = job.xpath(f"./div[{j}]/div[1]/div[2]/div[1]/a/text()")[0]
                    company_name.append(name)
            # print(company_name)
            for j in range(len(company_name)):
                company_name[j] = company_name[j].split()[0]
            # print(company_name)
            company_names.extend(company_name)
            # print(company_names)

            # 公司性质、规模、类型
            company_nature = []
            company_size = []
            company_type = []
            for j in range(1, len(career_name) + 1):
                nature_size_type = job.xpath(f"./div[{j}]/div[1]/div[2]/div[2]/*/text()")
                # print(nature_size_type)
                if len(nature_size_type) == 3:
                    nature = nature_size_type[0]
                    size = nature_size_type[1]
                    type_ = nature_size_type[2]
                    company_nature.append(nature)
                    company_size.append(size)
                    company_type.append(type_)
                elif len(nature_size_type) == 2:
                    size = nature_size_type[0]
                    type_ = nature_size_type[1]
                    company_nature.append("None")
                    company_size.append(size)
                    company_type.append(type_)
                elif len(nature_size_type) == 1:
                    size = nature_size_type[0]
                    company_nature.append("None")
                    company_type.append("None")
                    company_size.append(size)
            # print(company_nature)
            for j in range(len(company_nature)):
                company_nature[j] = company_nature[j].split()[0]
                company_size[j] = company_size[j].split()[0]
                company_type[j] = company_type[j].split()[0]
            # print(company_nature)
            company_natures.extend(company_nature)
            company_sizes.extend(company_size)
            company_types.extend(company_type)
            # print(company_natures)

    for salary_index in range(len(career_salaries)):
        max_salary = 0
        min_salary = 0
        try:
            if career_salaries[salary_index] != "面议":
                if "千" in career_salaries[salary_index].split("-")[1]:
                    min_salary = career_salaries[salary_index].split("-")[0].strip("千")
                    max_salary = career_salaries[salary_index].split("-")[1].split("千")[0]
                    if "." in min_salary:
                        min_salary = int(min_salary.split(".")[0] + min_salary.split(".")[1])
                    else:
                        min_salary = int(min_salary)
                    if "." in max_salary:
                        max_salary = int(max_salary.split(".")[0] + max_salary.split(".")[1])
                    else:
                        max_salary = int(max_salary)
                    # print(min_salary)
                    # print(max_salary)
                elif "万" in career_salaries[salary_index].split("-")[0]:
                    min_salary = career_salaries[salary_index].split("-")[0].strip("万")
                    max_salary = career_salaries[salary_index].split("-")[1].split("万")[0]
                    if "." in min_salary:
                        min_salary = int(min_salary.split(".")[0] + min_salary.split(".")[1])
                    else:
                        min_salary = int(min_salary) * 10
                    if "." in max_salary:
                        max_salary = int(max_salary.split(".")[0] + max_salary.split(".")[1])
                    else:
                        max_salary = int(max_salary) * 10
                    # print(min_salary)
                    # print(max_salary)
                elif "千" in career_salaries[salary_index].split("-")[0] and "万" in \
                        career_salaries[salary_index].split("-")[1]:
                    min_salary = career_salaries[salary_index].split("-")[0].split("千")[0]
                    max_salary = career_salaries[salary_index].split("-")[1].split("万")[0]
                    if "." in min_salary:
                        min_salary = int(min_salary.split(".")[0] + min_salary.split(".")[1])
                    else:
                        min_salary = int(min_salary)
                    if "." in max_salary:
                        max_salary = int(max_salary.split(".")[0] + max_salary.split(".")[1])
                    else:
                        max_salary = int(max_salary) * 10
                mean_salary = (max_salary + min_salary) / 2
                career_mean_salaries.append(mean_salary)
            else:
                career_mean_salaries.append("None")
        except ValueError:
            career_mean_salaries.append("None")

    # 保存数据到csv/excel
    # print(len(career_names))
    # print(len(career_salaries))
    # print(len(career_educations))
    # print(len(career_exps))
    # print(len(company_types))
    # print(len(company_names))
    # print(len(company_sizes))
    # print(len(company_natures))
    filename = filepath  # 选择一个文件名
    with open(filename, 'w', newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(
            ['岗位名称', '岗位薪资', '岗位要求学历', '岗位要求经验', '公司名称', '公司规模', '公司类型', '公司性质', '平均薪资'])  # 写入表头
        for x in range(len(career_names)):
            writer.writerow(
                [career_names[x], career_salaries[x], career_educations[x], career_exps[x], company_names[x], company_sizes[x],
                 company_types[x], company_natures[x], career_mean_salaries[x]])

    print("获取数据完毕")


if __name__ == '__main__':
    spider_information("南京", "厨师", "../data/career_data.csv")
